phase1: add support for overriding feeds host

Currently git.openwrt.org is sometimes overloaded, so lets implement a
mechanism to override the source host for feeds.

This introduces a `feeds_host_override` configuration option. When set,
the buildmaster will temporarily modify `feeds.conf.default` to point to
the alternate host (e.g., GitHub) before updating feeds, and restore the
original configuration afterwards.

This is particularly useful when the primary git server is returning 503
errors:

    Updating feed 'packages' from 'https://git.openwrt.org/feed/packages.git' ...
    Cloning into './feeds/packages'...
    fatal: unable to access 'https://git.openwrt.org/feed/packages.git/': The requested URL returned error: 503
    failed.

Signed-off-by: Petr Štetiar <[email protected]>

diff --git a/docker/config.ini b/docker/config.ini
index 9da83eb3580ab36189ce107f6cfbc02f450ff60a..8daa00edb53727af0c0175e9f050f30a61645ba7 100644 (file)
--- a/docker/config.ini
+++ b/docker/config.ini
@@ -10,6 +10,7 @@ status_password = admin
 buildbot_url = http://buildmaster-phase1:8010/
 expire = 1209600
 port = ssl:9989:privateKey=/certs/master.key:certKey=/certs/master.crt
+feeds_host_override = github.com/openwrt
 config_seed = # Seed configuration
        CONFIG_BUILDBOT=y
        CONFIG_DEVEL=y

diff --git a/phase1/config.ini.example b/phase1/config.ini.example
index 455507e50fc2d4970e1567d4a910c8acdfea4f80..79cb44f952f40d368f123905040359d05bd23f29 100644 (file)
--- a/phase1/config.ini.example
+++ b/phase1/config.ini.example
@@ -9,6 +9,7 @@ status_bind = tcp:8010:interface=127.0.0.1
 status_user = example
 status_password = example
 port = 9989
+feeds_host_override =
 
 [irc]
 host = irc.freenode.net

diff --git a/phase1/master.cfg b/phase1/master.cfg
index a2870dafdee6685434f329a713989b2cd4ac36a1..9967eafa8d0c0a3d42ae7af9819dc2a68471e936 100644 (file)
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -107,6 +107,7 @@ def ini_parse_branch(section):
 
 # PB port can be either a numeric port or a connection string
 pb_port = inip1.get("port") or 9989
+feeds_host_override = inip1.get("feeds_host_override", "").strip()
 
 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
@@ -609,6 +610,15 @@ def IsRemoteShaSumsAvailable(step):
     return step.getProperty("have_remote_shasums")
 
 
+def IsFeedsHostOverrideEnabled(step):
+    return bool(feeds_host_override)
+
+
[email protected]
+def GetFeedsHostOverride(props):
+    return feeds_host_override
+
+
 def GetBaseVersion(branch):
     if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
         return branch.split("-")[1]
@@ -986,6 +996,37 @@ def prepareFactory(target):
         )
     )
 
+    factory.addStep(
+        ShellCommand(
+            name="feeds-backup",
+            description="Backing up feeds.conf.default",
+            descriptionDone="feeds.conf.default backed up",
+            command=["cp", "-p", "feeds.conf.default", "feeds.conf.default.bak"],
+            doStepIf=IsFeedsHostOverrideEnabled,
+            haltOnFailure=True,
+        )
+    )
+
+    factory.addStep(
+        ShellCommand(
+            name="feeds-override",
+            description="Overriding feeds host",
+            descriptionDone="Feeds host overridden",
+            command=[
+                "sed",
+                "-i",
+                "-E",
+                Interpolate(
+                    "s;git.openwrt.org/(feed|project);%(kw:host)s;",
+                    host=GetFeedsHostOverride,
+                ),
+                "feeds.conf.default",
+            ],
+            doStepIf=IsFeedsHostOverrideEnabled,
+            haltOnFailure=True,
+        )
+    )
+
     # feed
     factory.addStep(
         ShellCommand(
@@ -998,6 +1039,17 @@ def prepareFactory(target):
         )
     )
 
+    factory.addStep(
+        ShellCommand(
+            name="feeds-restore",
+            description="Restoring feeds.conf.default",
+            descriptionDone="feeds.conf.default restored",
+            command="test -f feeds.conf.default.bak && mv -f feeds.conf.default.bak feeds.conf.default || true",
+            doStepIf=IsFeedsHostOverrideEnabled,
+            haltOnFailure=True,
+        )
+    )
+
     # feed
     factory.addStep(
         ShellCommand(